Import Libraries¶
In [0]:
import matplotlib.pyplot as plt
import matplotlib as mpl
from matplotlib.pyplot import imshow
mpl.rcParams['figure.figsize'] = (15,15)
mpl.rcParams['axes.grid'] = False
import numpy as np
from PIL import Image
import time
In [0]:
try:
%tensorflow_version 2.x
except Exception:
pass
import tensorflow as tf
In [0]:
#authorization from google drive
from google.colab import drive
drive.mount('/content/drive')
Download VGG19¶
In [0]:
#print all layers of VGG 19
vgg = tf.keras.applications.VGG19(include_top=False, weights='imagenet')
for layer in vgg.layers:
print(layer.name)
Decide Layers to Use from VGG19¶
According to the paper
content layer: higher layers in the network capture the high-level content in terms of object and their arrangement in the input image
style layer: require multiple layers and include the feature correlations
In [0]:
content_layers = ["block5_conv2"]
style_layers = ["block1_conv1","block2_conv1","block3_conv1","block4_conv1","block5_conv1"]
Define Content Loss and Style Loss¶
In [0]:
style_weight = 1e3
content_weight = 1e1
num_content_layers = 1
num_style_layers = 5
In [0]:
def content_loss(content_output, content_target):
content_loss = 0.5*tf.add_n([tf.reduce_mean((content_output-content_target)**2)])
return content_loss
In [0]:
def gram_matrix(style_output):
result = tf.linalg.einsum('aijk,aijl->akl', style_output, style_output)
return result
In [0]:
def style_loss(style_outputs, style_targets):
style_loss = tf.add_n([tf.reduce_mean((style_outputs[i]-style_targets[i])**2/(4*(style_outputs[i].shape[1]**2)*(style_targets[i].shape[1]**2)))
for i in range(num_style_layers)])
return style_loss
In [0]:
def total_loss(c_loss, s_loss):
return c_loss * content_weight / num_content_layers + s_loss * style_weight / num_style_layers
Generate Content and Style Representations¶
In [0]:
def get_vgg_layers(layers):
#freezing weights in vgg19 layers
vgg.trainable = False
output_layers = []
for layer in layers:
output_layers.append(vgg.get_layer(layer).output)
return tf.keras.Model([vgg.input], output_layers)
In [0]:
def content_representation(image, layers = content_layers):
output = get_vgg_layers(layers)
return output(image)[0]
In [0]:
def style_representation(image, layers = style_layers):
output = get_vgg_layers(layers)
output = output(image)
output = [gram_matrix(i) for i in output]
return output
In [0]:
def get_representation(image):
image_processed = tf.keras.applications.vgg19.preprocess_input(image*255.0)
content_output = content_representation(image)
style_output = style_representation(image)
return [content_output, style_output]
Gradient Descent¶
In [0]:
#pixel between 0 and 1
def clip(image):
return tf.clip_by_value(image, clip_value_min = 0.0, clip_value_max = 1.0)
opt = tf.optimizers.Adam(learning_rate=0.02, beta_1=0.99, epsilon=1e-1)
In [0]:
def gradient_descent(image):
with tf.GradientTape() as tape:
output = get_representation(image)
c_loss = content_loss(output[0], content_target)
s_loss = style_loss(output[1], style_target)
loss = total_loss(c_loss, s_loss)
grad = tape.gradient(loss, image)
opt.apply_gradients([(grad, image)])
image.assign(clip(image))
return [c_loss, s_loss, loss]
Run Model¶
In [0]:
#load and resize images
LENGTH = 480
WIDTH = 640
def load_and_resize_image(path_to_file):
image = plt.imread(path_to_file)
image = tf.image.convert_image_dtype(image, tf.float32)
image = tf.expand_dims(image,0)
image = tf.compat.v1.image.resize_bilinear(image, [LENGTH, WIDTH])
return image
In [0]:
!ls "/content/drive/My Drive/5242FinalProject"
In [0]:
#display content and style images
content_path = "/content/drive/My Drive/5242FinalProject/village.jpg"
style_path = "/content/drive/My Drive/5242FinalProject/wind.jpg"
content_image = load_and_resize_image(content_path)
style_image = load_and_resize_image(style_path)
plt.subplot(1, 2, 1)
imshow(tf.squeeze(content_image, axis=0))
plt.title('content')
plt.subplot(1, 2, 2)
imshow(tf.squeeze(style_image, axis=0))
plt.title('style')
plt.show()
In [0]:
#content and style representation of targets
content_target = get_representation(content_image)[0]
style_target = get_representation(style_image)[1]
Effect of different training iterations¶
In [0]:
tf.random.set_seed(0)
import time
start = time.time()
loss_ls = [[], [], []]
total_steps = 1000
#initialize with content image
image = tf.Variable(content_image)
for i in range(1, total_steps+1):
if i == 50 or i % 100 == 0:
print("Step " + str(i))
plt.subplot(1, 2, 1)
imshow(tf.squeeze(image, axis=0))
plt.show()
print(".", end='')
loss_result = gradient_descent(image)
for i in range(3):
loss_ls[i].append(loss_result[i].numpy())
end = time.time()
print("Total time: {:.1f}".format(end-start))
In [0]:
mpl.rcParams.update({'font.size': 12})
x = np.arange(1000)
plt.subplots_adjust(wspace = 0.3)
plt.subplot(2,2,1)
plt.plot(x, loss_ls[0], linewidth=2)
plt.xlabel('number of training iterations')
plt.ylabel('content_loss')
plt.xticks(np.arange(0, 1000, 100))
plt.axvline(x=200, color = 'r')
plt.axvline(x=450, color= 'r')
plt.subplot(2,2,2)
plt.plot(x, loss_ls[1], linewidth=2)
plt.xlabel('number of training iterations')
plt.ylabel('style_loss')
plt.xticks(np.arange(0, 1000, 100))
plt.axvline(x=50, color = 'r')
plt.axvline(x=400, color= 'r')
plt.subplot(2,2,3)
plt.plot(x, loss_ls[2], linewidth=2)
plt.xlabel('number of training iterations')
plt.ylabel('total_loss')
plt.xticks(np.arange(0, 1000, 100))
plt.axvline(x=50, color = 'r')
plt.axvline(x=400, color= 'r')
plt.show()
Effect of different random seeds¶
In [0]:
seeds = [0,23,625,2019]
loss_seeds = []
for seed in seeds:
print("seed " + str(seed))
tf.random.set_seed(seed)
import time
start = time.time()
loss_ls = [[], [], []]
total_steps = 1000
#initialize with content image
image = tf.Variable(content_image)
for i in range(1, total_steps+1):
if i % 200 == 0:
print("Step " + str(i))
plt.subplot(1, 2, 1)
imshow(tf.squeeze(image, axis=0))
plt.show()
print(".", end='')
loss_result = gradient_descent(image)
for i in range(3):
loss_ls[i].append(loss_result[i].numpy())
end = time.time()
print("Total time: {:.1f}".format(end-start))
loss_seeds.append(loss_ls)
In [0]:
#plot loss functions
mpl.rcParams.update({'font.size': 12})
x = np.arange(1000)
plt.subplots_adjust(wspace = 0.3)
plt.subplot(2,2,1)
plt.plot(x, loss_seeds[0][2], linewidth=2, label = 'seed 0')
plt.plot(x, loss_seeds[1][2], linewidth=2, label = 'seed 23')
plt.plot(x, loss_seeds[2][2], linewidth=2, label = 'seed 625')
plt.plot(x, loss_seeds[3][2], linewidth=2, label = 'seed 2019')
plt.legend(loc="upper right")
plt.xlabel('number of training iterations')
plt.ylabel('total_loss')
plt.xticks(np.arange(0, 1000, 100))
plt.show
Out[0]:
Effect of different relative weight of content & loss in the loss function¶
Can be completed by changing values of the variables style_weight and content_weight
Neural style transfer with two style images¶
In [0]:
#define a new total loss function that takes one content loss and two style losses
def total_loss_two_styles(c_loss, s1_loss, s2_loss):
return c_loss * content_weight / num_content_layers + s1_loss/2 * style_weight / num_style_layers + s2_loss/2 * style_weight / num_style_layers
In [0]:
#define a new gradient descent method
def gradient_descent_two_styles(image):
with tf.GradientTape() as tape:
output = get_representation(image)
c_loss = content_loss(output[0], content_target)
s1_loss = style_loss(output[1], style1_target)
s2_loss = style_loss(output[1], style2_target)
loss = total_loss_two_styles(c_loss, s1_loss, s2_loss)
grad = tape.gradient(loss, image)
opt.apply_gradients([(grad, image)])
image.assign(clip(image))
return [c_loss, s1_loss, s2_loss, loss]
In [0]:
content_path = "/content/drive/My Drive/5242FinalProject/village.jpg"
style1_path = "/content/drive/My Drive/5242FinalProject/wind.jpg"
style2_path = "/content/drive/My Drive/5242FinalProject/style_mosaic.jpg"
content_image = load_and_resize_image(content_path)
style1_image = load_and_resize_image(style1_path)
style2_image = load_and_resize_image(style2_path)
plt.subplot(2, 2, 1)
imshow(tf.squeeze(content_image, axis=0))
plt.title('content')
plt.subplot(2, 2, 2)
imshow(tf.squeeze(style1_image, axis=0))
plt.title('style')
plt.subplot(2, 2, 3)
imshow(tf.squeeze(style2_image, axis=0))
plt.title('style')
plt.show()
In [0]:
#content and style representation of targets
content_target = get_representation(content_image)[0]
style1_target = get_representation(style1_image)[1]
style2_target = get_representation(style2_image)[1]
In [0]:
tf.random.set_seed(0)
import time
start = time.time()
tf.random.set_seed(0)
import time
start = time.time()
loss_ls = [[], [],[],[]]
total_steps = 1000
#initialize with content image
image = tf.Variable(content_image)
for i in range(1, total_steps+1):
if i == 50 or i % 100 == 0:
print("Step " + str(i))
plt.subplot(1, 2, 1)
imshow(tf.squeeze(image, axis=0))
plt.show()
print(".", end='')
loss_result = gradient_descent_two_styles(image)
for i in range(4):
loss_ls[i].append(loss_result[i].numpy())
end = time.time()
print("Total time: {:.1f}".format(end-start))
Code Reference¶
1.Tutorial from Tensorflow: https://www.tensorflow.org/tutorials/generative/style_transfer